{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys; print('Python \\t\\t{0[0]}.{0[1]}'.format(sys.version_info))\n",
    "import tensorflow as tf; print('Tensorflow \\t{}'.format(tf.__version__))\n",
    "import keras; print('Keras \\t\\t{}'.format(keras.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline \n",
    "\n",
    "from sys import stdout\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"../mnist-data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist.train.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15,5))\n",
    "for i in list(range(10)):\n",
    "    plt.subplot(1, 10, i+1)\n",
    "    pixels = mnist.test.images[i]\n",
    "    pixels = pixels.reshape((28, 28))\n",
    "    plt.imshow(pixels, cmap='gray_r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set parameters\n",
    "learning_rate = 0.01\n",
    "training_iteration = 5\n",
    "batch_size = 250\n",
    "print_freq=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1, mean=0.0) #tf.constant(0.0, shape=shape)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TF graph input\n",
    "x = tf.placeholder('float', [None, 784]) # mnist data image of shape 28*28=784\n",
    "y = tf.placeholder('float', [None, 10]) # 0-9 digits recognition => 10 classes\n",
    "\n",
    "#dropout rate = 1 - keep_rate\n",
    "keep_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"reshape_0\") as scope:\n",
    "    input_x = tf.reshape(x, [-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# keras model for reference\n",
    "\n",
    "# model = Sequential()\n",
    "# model.add(Reshape(input_shape, input_shape=(784,)))\n",
    "\n",
    "# model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(MaxPooling2D((2,2)))\n",
    "\n",
    "# model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n",
    "# model.add(MaxPooling2D((2,2)))\n",
    "\n",
    "# model.add(Flatten())\n",
    "# model.add(Dropout(0.25))\n",
    "\n",
    "# model.add(Dense(256, activation='relu'))\n",
    "# model.add(Dropout(0.25))\n",
    "\n",
    "# model.add(Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"conv_1\") as scope:\n",
    "    \n",
    "    #filter definition\n",
    "    f_1 = weight_variable([3, 3, 1, 32])\n",
    "    b_1 = bias_variable([32])\n",
    "    \n",
    "    # Construct a dense linear model, with act=relu and dropout\n",
    "    layer_1 = tf.nn.relu(tf.nn.conv2d(input=input_x, filter=f_1, strides=[1,1,1,1], padding='SAME') + b_1)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"conv_2\") as scope:\n",
    "    \n",
    "    #filter definition\n",
    "    f_2 = weight_variable([3, 3, 32, 32])\n",
    "    b_2 = bias_variable([32])\n",
    "    \n",
    "    # Construct a dense linear model, with act=relu and dropout\n",
    "    layer_2 = tf.nn.relu(tf.nn.conv2d(input=layer_1, filter=f_2, strides=[1,1,1,1], padding='SAME') + b_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"maxpool_3\") as scope:\n",
    "    # maxpool strides of 2, size of pool is 2x2\n",
    "    layer_3 = tf.nn.max_pool(value=layer_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"conv_4\") as scope:\n",
    "    \n",
    "    #filter definition\n",
    "    f_4 = weight_variable([3, 3, 32, 64])\n",
    "    b_4 = bias_variable([64])\n",
    "    \n",
    "    # Construct a dense linear model, with act=relu and dropout\n",
    "    layer_4 = tf.nn.relu(tf.nn.conv2d(input=layer_3, filter=f_4, strides=[1,1,1,1], padding='SAME') + b_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"conv_5\") as scope:\n",
    "    \n",
    "    #filter definition\n",
    "    f_5 = weight_variable([3, 3, 64, 64])\n",
    "    b_5 = bias_variable([64])\n",
    "    \n",
    "    # Construct a dense linear model, with act=relu and dropout\n",
    "    layer_5 = tf.nn.relu(tf.nn.conv2d(input=layer_4, filter=f_5, strides=[1,1,1,1], padding='SAME') + b_5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"maxpool_6\") as scope:\n",
    "    # maxpool strides of 2, size of pool is 2x2\n",
    "    layer_6 = tf.nn.max_pool(value=layer_5, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"flatten_7\") as scope:\n",
    "    layer_7 = tf.reshape(layer_6, [-1, 7*7*64])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dropout_8\") as scope:\n",
    "    layer_8 = tf.nn.dropout(layer_7, keep_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dense_9\") as scope:\n",
    "\n",
    "    # Set model weights\n",
    "    W_9 = weight_variable([7*7*64, 256])\n",
    "    b_9 = bias_variable([256])\n",
    "\n",
    "    # Construct a dense linear model, with act=relu and dropout\n",
    "    layer_9 = tf.nn.relu(tf.matmul(layer_8, W_9) + b_9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dropout_10\") as scope:\n",
    "    layer_10 = tf.nn.dropout(layer_9, keep_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"dense_11\") as scope:\n",
    "\n",
    "    # Set model weights\n",
    "    W_11 = weight_variable([256, 10])\n",
    "    b_11 = bias_variable([10])\n",
    "\n",
    "    # Construct a dense linear model, with act=relu and dropout\n",
    "    layer_11 = tf.matmul(layer_10, W_11) + b_11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get the softmax as a separate tensorflow op\n",
    "y_out = tf.nn.softmax(layer_11)\n",
    "\n",
    "# softmax cross entropy descend on y_hat\n",
    "y_hat = layer_11\n",
    "\n",
    "# More name scopes will clean up graph representation\n",
    "with tf.name_scope(\"cost_function\") as scope:\n",
    "    # Minimize error using cross entropy\n",
    "    # Cross entropy\n",
    "    cost_function = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=y_hat))\n",
    "    \n",
    "    # Create a summary to monitor the cost function\n",
    "    tf.summary.scalar(\"cost_function\", cost_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"train\") as scope:\n",
    "    # Gradient descent\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = tf.equal(tf.argmax(y_out, 1), tf.argmax(y, 1))\n",
    "acc = tf.reduce_mean(tf.cast(predictions, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Merge all summaries into a single operator\n",
    "merged_summary_op = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Launch the graph\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# Logs and graph for tensorboard\n",
    "summary_writer = tf.summary.FileWriter('./tensorboard/tf-initial', graph=sess.graph)\n",
    "\n",
    "# Init the session\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Training cycle\n",
    "for iteration in range(training_iteration):\n",
    "    total_batch = int(mnist.train.num_examples/batch_size)\n",
    "    # Loop over all batches\n",
    "    avg_loss =0.\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        \n",
    "        # dropout placeholder\n",
    "        batch_kr = 0.75\n",
    "        \n",
    "        # Fit training using batch data\n",
    "        loss, accuracy, optm = sess.run([cost_function,acc,optimizer], \n",
    "            feed_dict={x: batch_xs, keep_rate: batch_kr, y: batch_ys})\n",
    "        \n",
    "        avg_loss += loss\n",
    "        \n",
    "        stdout.write('\\r{}/{} avg_cost:{:6f} cost:{:6f} acc:{:6f}'.format(i*batch_size, \n",
    "                                                                  mnist.train.num_examples, \n",
    "                                                                  avg_loss/(i+1), \n",
    "                                                                  loss, accuracy))\n",
    "        stdout.flush() \n",
    "            \n",
    "    # Display logs per iteration step\n",
    "    if iteration % print_freq ==0 :\n",
    "        accuracy_test = sess.run([acc], feed_dict={x: mnist.test.images, keep_rate: 1.0, y: mnist.test.labels})\n",
    "        print(\" epoch: {:02d} acc_test={:.9f}\".format(iteration, accuracy_test[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Accuracy:\", acc.eval({x: mnist.test.images, keep_rate:1.0, y: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test item #100 is a six\n",
    "pixels = mnist.test.images[100]\n",
    "\n",
    "#predict\n",
    "result = sess.run(y_out, feed_dict={x:[pixels], keep_rate:1.0})\n",
    "dict(zip(range(10), result[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_render(pixels, result, truth):\n",
    "    #pixels, result and truth are np vectors\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.subplot(1, 2, 1)\n",
    "    pixels = pixels.reshape((28, 28))\n",
    "    plt.imshow(pixels, cmap='gray_r')\n",
    "\n",
    "    plt.subplot(1, 2, 2)\n",
    "    \n",
    "    #index, witdh\n",
    "    ind = np.arange(len(result))\n",
    "    width = 0.4\n",
    "\n",
    "    plt.barh(ind,result, width, color='gray')\n",
    "    plt.barh(ind+width,truth,width, color='green')\n",
    "    plt.yticks(ind+width, range(10))\n",
    "    plt.margins(y=0)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "i = random.randint(0,mnist.test.images.shape[0])\n",
    "\n",
    "pixels = mnist.test.images[i]\n",
    "truth  = mnist.test.labels[i]\n",
    "result = sess.run(y_out, feed_dict={x:[pixels], keep_rate:1.0})[0]\n",
    "\n",
    "test_render(pixels, result, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### What went wrong?\n",
    "pixels = mnist.test.images\n",
    "truth = mnist.test.labels\n",
    "\n",
    "feed_dict = {x:pixels,keep_rate:1.0}\n",
    "result = sess.run(y_out, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_correct = result.argmax(axis=1) == truth.argmax(axis=1)\n",
    "incorrect = np.argwhere(index_correct==False).flatten()\n",
    "\n",
    "print(\"Incorrect predictions: {}\".format(len(incorrect)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,5))\n",
    "plt_idx = 1\n",
    "for i in list(incorrect[:16]):\n",
    "    plt.subplot(1, 16, plt_idx)\n",
    "    pixels = mnist.test.images[i]\n",
    "    pixels = pixels.reshape((28, 28))\n",
    "    plt.imshow(pixels, cmap='gray_r')\n",
    "    plt_idx += 1\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i = random.choice(list(incorrect))\n",
    "\n",
    "pixels = mnist.test.images[i]\n",
    "truth  = mnist.test.labels[i]\n",
    "\n",
    "feed_dict = {x:[pixels]}\n",
    "feed_dict.update({keep_rate:1.0})\n",
    "result = sess.run(y_out, feed_dict=feed_dict)[0]\n",
    "\n",
    "test_render(pixels, result, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Close the Session when we're done.\n",
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
